'''
Created on 2009-11-3

@author: wangyongtao
'''
from google.appengine import runtime
from google.appengine.runtime import apiproxy_errors
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.api import images

import mokomodel
import spider
import logging

_DEBUG = True
_PER_CRAWLER_MODEL_COUNT = 6
class CronPage(webapp.RequestHandler):
    def get(self, action):
        action = action.lower()
        logging.debug('action: %s'%action)
        if action == 'model':
            self.crawlermodel()
        elif action == 'avatar':
            self.cronavatarphotos()
        elif action == 'gallery':
            self.crongallery()
        elif action == 'album':
            self.cronalbum()
        else:
            logging.info('unknown action')
            l = mokomodel.cronlog(action='unkown action', content = action, successed = False)
            l.put()
        
    def crawlermodel(self):
        models = mokomodel.cronlog.get_non_crawled_models(_PER_CRAWLER_MODEL_COUNT)
        s = spider.spider()
        try:
            for m in models:
                logging.info('crawler model:%s'%m.username)
                s.crawler(m.homepage, mokomodel.model, True)
                mokomodel.cronlog.add_crawler_model(m.username, True)
        except runtime.DeadlineExceededError, e:
            logging.error('cron task: crawler model has met deadline exceeded error')
            
    def cronavatarphotos(self):
        models = mokomodel.cronlog.get_non_crawled_avatars(_PER_CRAWLER_MODEL_COUNT)
        try:
            for m in models:
                logging.debug('fetch model avatar: %s'%m.avatar)
                spider.fetch_avatar(m)
        except runtime.DeadlineExceededError, e:
            logging.error('cron task: fetch avatar has met deadline exceeded error:%s'%e.message)
            
    def crongallery(self):
        models = mokomodel.cronlog.get_non_crawled_gallery(_PER_CRAWLER_MODEL_COUNT)
        logging.info('models count: %d'%len(models))
        try:
            for m in models:
                logging.debug('Crawler gallery:%s'%m.username)
                result = spider.fetch_gallery(m)
                mokomodel.cronlog.add_crawler_gallery(m.username, result)
        except runtime.DeadlineExceededError, e:
            logging.error('cron task: fetch gallery has met deadline exceeded error:%s'%e.message)
    
    def cronalbum(self):
        albums = mokomodel.cronlog.get_non_crawled_albums(_PER_CRAWLER_MODEL_COUNT)
        logging.debug('albums count: %d'%len(albums))
        try:
            for a in albums:
                logging.debug('Crawler album:%s'%a.title)
                result = spider.fetch_album(a)
                mokomodel.cronlog.add_crawler_album(a.url, result)
        except runtime.DeadlineExceededError, e:
            logging.error('cron task: fetch album has met deadline exceeded error:%s'%e.message)
                
application = webapp.WSGIApplication([
            (r'/cron/(.*)', CronPage),
            ], debug=_DEBUG)

def main():
    run_wsgi_app(application)

if __name__ == "__main__":
    main()
        